In [1372]:
#Loading the libraries
!pip install matplotlib.backends.backend_pdf
import pandas as pd #For general purpose Data Manipulation
import numpy as np #For some operations - creating vectors and matrices/log transformation
from matplotlib.backends.backend_pdf import PdfPages #Plotting Box-Whisker
from matplotlib import pyplot as plt #Plotting Box-Whisker --> Outlier Detection
import seaborn as sns #Plotting Box-Whisker and others
from sklearn.model_selection import train_test_split #For Building Train and Test Set
from sklearn.linear_model import LinearRegression #For Regression Analysis
from sklearn.preprocessing import LabelEncoder #for the columns who will suffer from curse of dimensionality
from sklearn.preprocessing import OneHotEncoder #with less number of columns
from sklearn.preprocessing import StandardScaler
from sklearn.preprocessing import MinMaxScaler
In [566]:
train=pd.read_csv("D:/Assignment1_PGD B7/Train.csv")
train.head()
Out[566]:
In [729]:
X=train.iloc[:,:-1]
Y=train["Item_Outlet_Sales"]
In [697]:
train.head()
Out[697]:
In [764]:
Y.head()
Y.shape
Out[764]:
In [699]:
#Calculating the shape of original train data set
train.shape
Out[699]:
In [684]:
#Calculating the shape of X
X.shape
Out[684]:
In [490]:
#Calulating the shape of Y
Y.shape
Out[490]:
In [491]:
!pip install pandas_profiling
In [700]:
# Creating a new dataset with only numeric values
numeric_var_names=[key for key in dict(train.dtypes) if dict(train.dtypes)[key] in ['float64', 'int64', 'float32', 'int32']]
cat_var_names=[key for key in dict(train.dtypes) if dict(train.dtypes)[key] in ['object', 'O']]
print(numeric_var_names)
print(cat_var_names)
In [701]:
train_num=train[numeric_var_names]
train_num.head(5)
Out[701]:
In [702]:
train_cat=train[cat_var_names]
train_cat.head(5)
Out[702]:
In [703]:
# Getting Quick Summary
def quick_summary(x):
return pd.Series([x.count(), x.isnull().sum(), x.value_counts()],
index=['N', 'NMISS', 'ColumnsNames'])
num_summary=train_num.apply(lambda x: quick_summary(x))
num_summary
Out[703]:
In [704]:
train_num.describe()
Out[704]:
In [705]:
bp = PdfPages('x.pdf')
for num_variable in numeric_var_names:
fig,axes = plt.subplots(figsize=(10,4))
sns.boxplot( x=num_variable, data = train_num)
plt.title(str('Box Plot of ') + str(num_variable))
bp.savefig(fig)
bp.close()
Treatment of Skewness¶
In [706]:
train.skew()
Out[706]:
In [1423]:
"""This skewness of data needs to be removed by applying transformations"""
import numpy as np
train['Item_Weight'] = np.log(train['Item_Weight'])
train['Item_Weight'] = np.sqrt(train['Item_Weight'])
train['Item_Visibility'] = np.sqrt(train['Item_Visibility'])
train['Item_MRP'] = np.sqrt(train['Item_MRP'])
train['Outlet_Establishment_Year'] = np.square(train['Outlet_Establishment_Year'])
train['Item_Outlet_Sales'] = np.sqrt(train['Item_Outlet_Sales'])
train.skew()
train.shape
Out[1423]:
In [ ]:
"""Skewness is removed after treatment of skewness. Skewness can cause outliers thus it is important that outliers must get
removed"""
Treatment of Outliers¶
In [736]:
def outlier_capping(x):
ul = x.quantile(0.75)+1.5*(x.quantile(0.75)-x.quantile(0.25))
ll = x.quantile(0.25)-1.5*(x.quantile(0.75)-x.quantile(0.25))
x = x.clip_upper(ul)
x = x.clip_lower(ll)
return x
train_num=train_num.apply(lambda x: outlier_capping(x))
train_num.describe()
train.update(train_num)
train.head()
"""Outliers in X and Y both have been removed."""
Out[736]:
Plotting of Boxplot i.e. Categorical Data Vs Categorical Data¶
In [739]:
x = train[["Item_Weight","Item_Visibility"]].values
plt.boxplot(x)
plt.show()
In [740]:
y=train[["Item_MRP","Outlet_Establishment_Year","Item_Outlet_Sales"]].values
plt.boxplot(y)
plt.show()
In [504]:
import pandas_profiling as pd_prof
pd_prof.ProfileReport(train)
Out[504]:
In [711]:
pd_prof.ProfileReport(train_cat)
Out[711]:
In [507]:
pd_prof.ProfileReport(train_num)
Out[507]:
In [741]:
train.info()
In [742]:
X.update(train)
X.info()
In [743]:
train.describe()
Out[743]:
In [744]:
'''missing value imputation of numerical column "Item_Weight by mean"'''
mean=np.mean(train["Item_Weight"])
train["Item_Weight"].fillna(mean,inplace=True)
train.head()
Out[744]:
In [1143]:
'''missing value imputation of categorical column "Item_Size" by mode'''
train["Outlet_Size"].mode()
train["Outlet_Size"].fillna("Medium",inplace=True)
train["Outlet_Size"].head()
train.head()
Out[1143]:
In [1514]:
X=train.iloc[:,:-1]
Y=train["Item_Outlet_Sales"]
X.drop(["Item_Identifier"],axis=1)
Out[1514]:
TREATMENT OF DATA¶
1. As number of values in the each column of train dataset is too high, it causes a danger of "Curse of Dimensionality" which in turn impacts "Accuracy Level" henceforth we should use "Label Encoding" in columns which have many columns and one hot encoding or get_dummies in columns which have less number of columns.¶
2. Also one-hot encoding is performed on nominal data and label-encoding is performed on ordinal data.¶
3. One-Hot Encoding or get_dummies for X¶
4. After concating all the dummy variables with the original dataframe, we need to drop the column from which dummy variable has been created and also one dummy variable column because "Multicollinearity" condition" is created,whichinduces "Dummy Variable Trap" which can crash computers and also affect the accuracy to an undesirable extent.¶
In [1512]:
x_dummies=pd.get_dummies(X[["Item_Type","Outlet_Identifier"]]) #Use of get_dummies instead of One-Hot Encoding
x_dummies.head()
le=LabelEncoder()
X["Outlet_Size"]=le.fit_transform(X["Outlet_Size"])
X["Outlet_Location_Type"]=le.fit_transform(X["Outlet_Location_Type"])
X["Outlet_Type"]=le.fit_transform(X["Outlet_Type"])
X["Item_Fat_Content"]=le.fit_transform(X["Item_Fat_Content"])
x_dummies=pd.get_dummies(X)
x_dummies.head()
Out[1512]:
In [1511]:
x_dummies.skew()
Out[1511]:
Analysis of Data¶
Univariate Analysis¶
In [1516]:
# Lets check if the Item_Weight and Item_Visibility is normally distributed
# Lets first check overall Item_Weight and Item_Visibility
# Item_Weight distribution
from matplotlib import pyplot as plt
plt.hist(x_dummies['Item_Weight'])
plt.xlabel('Item_Weight')
plt.ylabel('Number of Items')
plt.show()
plt.hist(x_dummies['Item_Visibility'])
plt.xlabel('Item_Visibility')
plt.ylabel('Number of Items')
plt.show()
plt.hist(x_dummies['Item_MRP'])
plt.xlabel('Item_MRP')
plt.ylabel('Number of Items')
plt.show()
plt.hist(x_dummies['Outlet_Establishment_Year'])
plt.xlabel('Outlet_Establishment_Year')
plt.ylabel('Number of Items')
plt.show()
None of the data in the numerical columns or feature columns is normally distributed. Thus, to make this data normally distributed, we have to apply standard scaler normalization.¶
Bivariate Analysis¶
Numerical column vs numerical column :¶
Analysis that can be performed :
- Relationship analysis : Scatter plot
- Correlation analysis : Correlation matrix and plot
- Regression analysis : Simple linear regression (SLR)
Relationship analysis- Scatter Plot¶
In [1148]:
plt.scatter(x_dummies["Item_Weight"],x_dummies["Item_Visibility"],color='violet')
plt.xlabel("Item_Weight")
plt.ylabel("Item_Visibility")
plt.title("Item_Weight Vs Item_Visibility")
plt.show()
In [1149]:
plt.scatter(x_dummies["Item_MRP"],x_dummies["Outlet_Establishment_Year"],color='orange')
plt.xlabel("Item_MRP")
plt.ylabel("Outlet_Establishment_Year")
plt.title("Item_MRP Vs Outlet_Establishment_Year")
plt.show()
Correlation Plot- Correlation Matrix and Plot¶
In [1151]:
matrix=x_dummies.corr()
matrix
Out[1151]:
In [1152]:
plt.rcParams['figure.figsize'] = [10,10]
from matplotlib import pyplot as plt
plt.matshow(matrix,cmap='Blues')
plt.show()
"""Stripplot"""¶
In [1194]:
import seaborn as sns
sns.stripplot(x_dummies['Item_Weight'],x_dummies['Item_MRP'],jitter=0.3,size=4)
plt.show()
x_dummies['Item_Weight'] = np.log(x_dummies['Item_Weight'])
x_dummies['Item_Weight'] = np.sqrt(x_dummies['Item_Weight'])
x_dummies['Item_Visibility'] = np.sqrt(x_dummies['Item_Visibility'])
x_dummies['Item_MRP'] = np.sqrt(x_dummies['Item_MRP'])
x_dummies['Outlet_Establishment_Year'] = np.square(x_dummies['Outlet_Establishment_Year'])
x_dummies['Item_Outlet_Sales'] = np.sqrt(x_dummies['Item_Outlet_Sales'])
train.skew()
train.shape
In [1193]:
sns.pairplot(train)
plt.show()
Categorical Column Vs Numerical Column¶
In [1192]:
import seaborn as sns
from matplotlib import pyplot as plt
g = sns.FacetGrid(train,col="Outlet_Size")
g = g.map(plt.hist,"Outlet_Establishment_Year")
plt.show()
import seaborn as sns
from matplotlib import pyplot as plt
g = sns.FacetGrid(train,col="Item_Fat_Content")
g = g.map(plt.hist,"Item_Weight")
plt.show()
import seaborn as sns
from matplotlib import pyplot as plt
g = sns.FacetGrid(train,col="Outlet_Size")
g = g.map(plt.hist,"Outlet_Establishment_Year")
plt.show()
Normalization Techniques¶
Use MinMaxScaler as the default if you are transforming a feature. Its non-disturbing.¶
You could use RobustScaler if you have outliers and want to reduce their influence.However, you might be better off removing the outliers, instead.¶
Use standardScaler if you need a relatively normal distribution.¶
Use Normalizer sparingly-if normalizes sample rows, not feature columns. It can use l2 or l1 normalization.¶
To know more about the distribution of data , we will do univariate analysis and bivariate analysis of data with plots such as Histogram.¶
In [1167]:
# Min-Max Scaler , Normalizing X"""
from sklearn.preprocessing import StandardScaler
sc=MinMaxScaler()
sc_data=sc.fit_transform(x_dummies)
sc_data=pd.DataFrame(sc_data)
sc_data.columns=x_dummies.columns
sc_data.head()
Out[1167]:
In [1168]:
Y.head()
Out[1168]:
Linear Regression¶
In [1471]:
"""Importing libraries and splitting the data"""
import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error
In [1526]:
x_train,x_test,y_train,y_test=train_test_split(x_dummies,Y,test_size=0.2,random_state=0)
reg = LinearRegression()
model1=reg.fit(x_train,y_train)
In [1518]:
len(x_train)
Out[1518]:
In [1519]:
len(y_train)
Out[1519]:
In [1527]:
"""Predicting Y_pred using x_test"""
Y_pred = reg.predict(x_test)
Y_pred
len(Y_pred)
Out[1527]:
In [1528]:
from sklearn.metrics import r2_score, mean_squared_error
rmse = np.sqrt(mean_squared_error(y_test,Y_pred))
rmse
Out[1528]:
In [1529]:
r2= r2_score(y_test,Y_pred) # Built-in function score() indicates R-Square value
print(r2*100)
pd.set_option("display.float_format",lambda x: "%.3f" % x)
In [1206]:
"""Plotting Residual Plots"""
x_axis = [i for i in range(1,len(Y_pred)+1)]
x_plot = plt.scatter(x_axis,(y_test-Y_pred),c='green')
plt.plot(x_axis,[0]*len(Y_pred),c='yellow')
plt.title('Residual Plot')
Out[1206]:
In [1563]:
x_dummies.head()
Out[1563]:
Lasso-Ridge Regularization¶
In [1564]:
from mpl_toolkits.mplot3d import Axes3D
from sklearn.linear_model import Ridge, Lasso
In [1565]:
x_train,x_test,y_train,y_test=train_test_split(x_dummies,Y,test_size=0.3,random_state=0)
print(len(x_train),len(y_train))
In [1566]:
reg=LinearRegression()
model=reg.fit(x_train,y_train)
In [1533]:
model.coef_
Out[1533]:
In [1567]:
model.intercept_
Out[1567]:
In [1568]:
y_pred=model.predict(x_test)
In [1569]:
from sklearn.metrics import r2_score, mean_squared_error
rmse=np.sqrt(mean_squared_error(y_test,y_pred))
r2=r2_score(y_test,y_pred)
print("RMSE=",rmse)
print("R2 score=",r2)
In [1570]:
#residual plot
x=[i for i in range(1,len(y_pred)+1)]
x_plot=plt.scatter(x,(y_pred - y_test),c='orange')
plt.plot(x,[0]*len(y_pred),c="blue")
plt.title('Residual plot')
Out[1570]:
Regularization Techniques¶
In [1537]:
predictors=x_train.columns
predictors
Out[1537]:
In [1538]:
coef=pd.Series(reg.coef_, predictors).sort_values()
coef
Out[1538]:
In [1539]:
coef.plot(kind="bar",title="Modal Coefficients")
Out[1539]:
In [1540]:
"""
alpha is hyperparameter of Ridge, which means that they are not automatically learned by the model instead they have to be set
manually.
"""
#alpha_range=[0.01,0.05,0.5,5]
ridgeReg=Ridge(alpha=0.01, normalize=True)
model=ridgeReg.fit(x_train,y_train)
pred=model.predict(x_test)
rmse=np.sqrt(mean_squared_error(y_test, pred))
r2=r2_score(y_test, pred)
print("RMSE=", rmse, "R2=", r2)
In [1541]:
df=pd.DataFrame(columns=['Linear Reg', "Ridge Reg"])
for i in range(len(ridgeReg.coef_)):
df=df.append({'Linear Reg': reg.coef_[i], "Ridge Reg":ridgeReg.coef_[i]},ignore_index=True)
df.head()
Out[1541]:
In [1495]:
predictors=x_train.columns
coef=pd.Series(ridgeReg.coef_,predictors).sort_values()
coef.plot(kind='bar',title='Model Coefficients')
print(coef)
"""It tells about what which independent variables hold much significance"""
Out[1495]:
In [1542]:
#alpha_range=[0.01,0.05,0.5,5]
lassoReg=Lasso(alpha=0.01, normalize=True)
model=lassoReg.fit(x_train,y_train)
pred=model.predict(x_test)
rmse=np.sqrt(mean_squared_error(y_test, pred))
r2=r2_score(y_test, pred)
print("RMSE=",rmse)
print("R-square=",r2)
co=lassoReg.coef_
inte=lassoReg.intercept_
In [1543]:
df=pd.DataFrame(columns=['Linear Reg',"Ridge Reg", 'Lasso Reg'])
for i in range(len(ridgeReg.coef_)):
df=df.append({'Linear Reg': reg.coef_[i],"Ridge Reg":ridgeReg.coef_[i],"Lasso Reg":lassoReg.coef_[i]},ignore_index=True)
df.head()
Out[1543]:
In [1544]:
print("RMSE=",rmse,"R2=",r2)
coef=pd.Series(lassoReg.coef_,predictors).sort_values()
print(coef)
coef.plot(kind="bar",title="Model Coefficients")
Out[1544]:
Analysis of Data¶
- (Before Regularization) Previous Model1=> RMSE=10.8201 , R2=65.13
- (Simple Regularization) Previous Model2(Using Alpha)=>RMSE=10.822043164075689 R2 score= 0.6523377799908223
- RMSE= 10.828437790653059 R2= 0.6519267990213873[L2 Regularization]
- RMSE= 11.000884418122189 R2= 0.6396102429778721[L1 Regularization]
- Clearly observing, L2 regularisation is better than the L2 regularisation and previous models because its accuracy is the maximum
- Regularization has a significant impact but it fails to increase the value of the R2 square, therefore it is not important to put Regularization on the data Independent Variables:
Item_Identifier,Item_Weight,Item_Fat_Content Item_Visibility,Item_Type,Item_MRP,Outlet_Identifier,Outlet_Establishment_Year,Outlet_Size,Outlet_Location_Type,Outlet_Type,Item_Outlet_Sales. "The impact on outlet sales are caused by the positive values of coef---":
Item_Weight 2.232 Item_Type_Soft Drinks 6.674 Item_Type_Meat 8.607 Item_Type_Breads 14.587 Item_MRP 14.724 Item_Fat_Content 33.438 Item_Type_Fruits and Vegetables 56.863 Item_Type_Others 65.276 Item_Type_Starchy Foods 70.085 Outlet_Identifier_OUT045 97.045 Outlet_Identifier_OUT013 139.639 Outlet_Identifier_OUT046 220.273 Outlet_Identifier_OUT049 245.044 Outlet_Identifier_OUT017 258.164 Item_Type_Seafood 304.190 Outlet_Type 389.815 Outlet_Identifier_OUT035 393.441 Outlet_Identifier_OUT027 707.095
"Item_Weight,Item_MRP,Outlet_Identifier, Item_Type, Outlet_Type, Item_Fat_Content"- Independent Variables which cause significant impact on the sales of outlet.
Analysis of Test Data¶
In [1545]:
test=pd.read_csv("D:/Assignment1_PGD B7/Test.csv")
test.head()
Out[1545]:
Treatment of Test.csv¶
Imputing of Missing Values¶
In [1546]:
test.isnull().sum()
Out[1546]:
In [1553]:
me=np.mean(test["Item_Weight"])
test["Item_Weight"].fillna(me,inplace=True)
test["Outlet_Size"].mode()
test["Outlet_Size"].fillna("Medium",inplace=True)
test["Outlet_Size"].head()
test.head()
test.shape
test.isnull().sum()
Out[1553]:
In [1554]:
'''Using Get_Dummies Function for nominal Data and Label Encoding
for Ordinal Data'''
import pandas as pd
test1=test.drop("Item_Identifier",axis=1)
test1.head()
Out[1554]:
In [1555]:
test_dummies=pd.get_dummies(test1[["Item_Type","Outlet_Identifier"]]) #Use of get_dummies instead of One-Hot Encoding
test_dummies.head()
Out[1555]:
In [1556]:
from sklearn.preprocessing import LabelEncoder
from sklearn.linear_model import LinearRegression
le=LabelEncoder()
test1["Outlet_Size"]=le.fit_transform(test1["Outlet_Size"])
test1["Outlet_Location_Type"]=le.fit_transform(test1["Outlet_Location_Type"])
test1["Outlet_Type"]=le.fit_transform(test1["Outlet_Type"])
test1["Item_Fat_Content"]=le.fit_transform(test1["Item_Fat_Content"])
test_dummies=pd.get_dummies(test1)
test_dummies.head()
test_dummies.shape
Y1=Y[0:5681]
In [1557]:
x_train,x_test,y_test,y_train=train_test_split(test_dummies,Y1,test_size=0.2,random_state=0)
In [1558]:
len(y_train)
predicted_values=model.predict(test_dummies)
print(predicted_values)
"""Predicted Values are:"""
Out[1558]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]: